﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;

namespace CloudXmlScanner
{
    class Program
    {
        // Needed to access the specific storage account and root container that we will parse through.
        private const string ConnectionString = "DefaultEndpointsProtocol=http;AccountName=YourAccountName;AccountKey=YourAccountKey";
        private const string RootContainerName = "RootContainerName";

        // Used to keep track of distinct XML elements, as well as the file(s) it is found in.
        private static readonly Dictionary<string, HashSet<string>> UnexpectedDictionary = new Dictionary<string, HashSet<string>>();

        // Used to track files that could not be parsed due to invalud XML.
        private static readonly Dictionary<string, string> ParseErrorDictionary = new Dictionary<string, string>(); 

        // Names we do not want to track because they are known to be valid
        private static readonly HashSet<string> KnownGoodList = new HashSet<string>()
        {
            "catalog",
            "book",
            "author",
            "title",
            "genre",
            "price",
            "publish_date",
            "description"
        };

        private static CloudBlobContainer _rootContainer;

        static void Main()
        {
            // Access the Storage account
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConnectionString);
            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
            _rootContainer = blobClient.GetContainerReference(RootContainerName);

            // Retrieve all XML and TXT blobs under this root as a flat list, ignoring the "virtual" directory structure.
            IEnumerable<CloudBlockBlob> xmlBlobs = _rootContainer
                .ListBlobs(null, true)
                .OfType<CloudBlockBlob>()
                .Where(blob => blob.Name.EndsWith(".xml") || blob.Name.EndsWith(".txt"));
            
            // Find the erroneous XML element names in each blob and merge them with the "global" dictionary.
            foreach(var blob in xmlBlobs)
            {
                XDocument xml = GetXDocumentFromBlob(blob);

                if (xml != null)
                {
                    IEnumerable<string> unexpected = FindUnexpectedNames(xml);
                    MergeUnexpectedNames(blob.Name, unexpected);
                }
            };

            OutputNames();
        }

        static XDocument GetXDocumentFromBlob(CloudBlockBlob blob)
        {
            // Download the contents of the blob into a stream
            var stream = _rootContainer.GetBlockBlobReference(blob.Name).OpenRead();
            stream.Seek(0, SeekOrigin.Begin);

            // First, attempt to parse the entire file to identify invalid xml.
            try
            {
                XDocument xml = XDocument.Load(stream);
                return xml;
            }
            catch (System.Xml.XmlException e)
            {
                // If there are any errors loading/parsing the XML, store a reference to the blob in a secondary dictionary.
                Console.WriteLine(e.Message);
                ParseErrorDictionary.Add(blob.Name, e.Message);
                return null;
            }
        }

        static void MergeUnexpectedNames(string blobName, IEnumerable<string> names)
        {
            // Merge the erroneous names into the UnexpectedDictionary.
            foreach (var name in names)
            {
                if (!UnexpectedDictionary.ContainsKey(name))
                {
                    // We aren't already tracking this name, so add it and the file it was found in.
                    UnexpectedDictionary.Add(name, new HashSet<string>() { blobName });
                }
                else
                {
                    // We are already tracking this name, so just record the file it was found in.
                    UnexpectedDictionary[name].Add(blobName);
                }
            }
        }

        private static IEnumerable<string> FindUnexpectedNames(XDocument xml)
        {

            var output = new HashSet<string>();
            // Iterate over the xml elements and ingest them into the output collection.
            foreach (XElement descendant in xml.Descendants())
            {
                // Retrieve the node name, including prefix if applicable.
                var nodeName = descendant.Name.NamespaceName == string.Empty
                    ? descendant.Name.LocalName
                    : string.Format("{0}:{1}", descendant.GetPrefixOfNamespace(descendant.Name.Namespace),
                        descendant.Name.LocalName);

                if (!KnownGoodList.Contains(nodeName))
                {
                    // If we aren't explicitly ignoring this name and we aren't already tracking it for this file, add it to the output.
                    if (!output.Contains(nodeName))
                    {
                        output.Add(nodeName);
                    }
                }
            }
            return output;
        }

        public static void OutputNames()
        {
            using (var file = new StreamWriter("output.txt", false))
            {

                // Output all excluded XML names as a reminder.
                if (KnownGoodList.Count > 0)
                {
                    file.WriteLine("Excluding known good node names:");

                    foreach (var exclusion in KnownGoodList)
                    {
                        file.WriteLine("    {0}", exclusion);
                    }
                }

                // Output all distinct keys found, along with their distinct values.
                file.WriteLine("\r\n\r\n{0} Unexpected node names:", UnexpectedDictionary.Count);
                foreach (var name in UnexpectedDictionary.Keys)
                {
                    file.WriteLine("\r\n    Name: {0}", name);
                    file.WriteLine("    Found in files:");
                    foreach (var value in UnexpectedDictionary[name])
                    {
                        file.WriteLine("        {0}", value);
                    }
                }

                // Output any parse errors
                if (ParseErrorDictionary.Count > 0)
                {
                    file.WriteLine("\r\n\r\nUnable to parse {0} files:", ParseErrorDictionary.Count);
                    foreach (var error in ParseErrorDictionary)
                    {
                        file.WriteLine("\r\n    File: {0}", error.Key);
                        file.WriteLine( "    Message: {0}", error.Value);
                    }
                }
            }
        }
    }
}
